package com.eric.downloader;

import java.io.BufferedReader;
import java.io.EOFException;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.ConnectException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.SocketException;
import java.net.SocketTimeoutException;
import java.net.URL;
import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLSession;

import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.LogFactory;
import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.RedirectException;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.config.RequestConfig.Builder;
import org.apache.http.client.entity.GzipDecompressingEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.apache.log4j.PropertyConfigurator;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Attribute;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.eric.Config;
import com.eric.utils.UrlUtils;
import com.eric.utils.page.ArticleExtractor;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.ProxyConfig;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;

/**
 * 根据返回码进行判断，如若网站限制爬虫 则模拟浏览器进行下载 基于httpclient封装
 * 
 * @author Eric
 *
 */
public class SimplePageDownloader {

	private String url, realUrl;

	private String charset;

	private String ua = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/52.0.2743.116 Chrome/52.0.2743.116 Safari/537.36";

	final CloseableHttpClient httpclient = HttpClients.custom().setUserAgent(ua)
			.setDefaultRequestConfig(RequestConfig.custom().setCookieSpec(CookieSpecs.STANDARD).build())
			.setRetryHandler(new RetryHandler()).disableAutomaticRetries().build();

	protected String html;

	protected Document doc;

	Logger logger;

	private int connectionTimeout = 2000;

	private int socketTimeout = 5000;

	private int status;

	protected MyProxy p;

	private String proxy;

	private int port;

	private int retry = 3;

	private boolean useBrowser = false;

	public static void main(String[] args) throws IOException {
		String s = "https://zhuanlan.zhihu.com/p/21629412";// TODO zhihu下载问题
		SimplePageDownloader spd = new SimplePageDownloader();
		System.out.println(spd.downloadByBrowse(s).document().text());
		// Document doc = spd.download(s).document();
		// MyProxy mp = new MyProxy();
		// spd.setProxy(mp);
		// SeoPage page = SeoPage.downloadSeoPage(s);
		//
		// for (String a : page.anchors()) {
		// if (a.length() > 10)System.out.println(a);
		// }

	}

	public SimplePageDownloader() {
		logger = LoggerFactory.getLogger(SimplePageDownloader.class);
		try {
			PropertyConfigurator.configure(new URL(Config.LOG4J_PROP));
		} catch (MalformedURLException e) {
			e.printStackTrace();
		}
		;
	}

	public SimplePageDownloader(MyProxy p) {
		logger = LoggerFactory.getLogger(SimplePageDownloader.class);
		try {
			PropertyConfigurator.configure(new URL(Config.LOG4J_PROP));
		} catch (MalformedURLException e) {
			e.printStackTrace();
		}
		this.p = p;
	}

	public SimplePageDownloader setTimeOut(int timeout) {
		this.connectionTimeout = timeout;
		return this;
	}

	public SimplePageDownloader useBrowser(boolean trueOrFalse) {
		this.useBrowser = trueOrFalse;
		return this;
	}

	/**
	 * 
	 * @return 文本形式的html
	 */
	public String html() {
		return html;
	}

	public String text() {
		return doc.text();
	}

	public String article() {
		ArticleExtractor ex = null;
		ex = new ArticleExtractor(document());
		return ex.extract();
	}

	public Document document() {
		if (doc == null && html != null) {
			doc = Jsoup.parse(html);
			return doc;
		} else {
			return doc;
		}
	}

	public String charset() {
		return charset;
	}

	public boolean redirected() {
		if (!url.equals(realUrl) && realUrl != null)
			return true;
		else
			return false;
	}

	public int status() {
		return status;
	}

	public String realUrl() {
		return (realUrl == null) ? url : realUrl;
	}

	/**
	 * 带协议的host
	 * 
	 * @return
	 */
	public String host() {
		URL url = null;
		String host = null;
		try {
			url = new URL(realUrl());
			host = url.getProtocol() + "://" + url.getHost();
		} catch (MalformedURLException e) {
			e.printStackTrace();
		}

		return host;
	}

	/**
	 * 输入关键词，并从搜索引擎获取页面
	 * 
	 * @param queryBaseUrl
	 *            搜索引擎查询参数
	 * @param kw
	 *            关键词
	 * @return 纯html形式的查询结果
	 */
	public String querySearchEngine(String queryBaseUrl, String kw) {
		try {
			if (useBrowser) {
				downloadByBrowse(queryBaseUrl + URLEncoder.encode(kw, "utf-8"));
			} else {
				download(queryBaseUrl + URLEncoder.encode(kw, "utf-8"));// 这里会做proxy判断
			}
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
		}
		return html;
	}

	public SimplePageDownloader setProxy(MyProxy p) {
		this.p = p;
		return this;
	}

	public void setProxy(String proxy, String port) {
		this.proxy = proxy;
		this.port = Integer.parseInt(port);
	}

	public boolean useProxy() {
		if (this.proxy != null || p != null)
			return true;
		else
			return false;
	}

	/**
	 * 使用htmlunit模拟浏览器行为下载网页
	 * 
	 * @return
	 */
	public SimplePageDownloader downloadByBrowse(String url) {
		this.url = UrlUtils.handleUrl(url);
		realUrl = realUrl(this.url);
		HtmlPage page = null;
		java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(java.util.logging.Level.OFF);
		LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log",
				"org.apache.commons.logging.impl.NoOpLog");
		try {
			final WebClient client = new WebClient(BrowserVersion.INTERNET_EXPLORER);

			if (useProxy()) {
				ProxyConfig pc = new ProxyConfig();
				pc.setProxyHost(proxy);
				pc.setProxyPort(port);
				client.getOptions().setProxyConfig(pc);
			}

			client.getOptions().setJavaScriptEnabled(true);
			client.getOptions().setCssEnabled(false);
			client.getOptions().setThrowExceptionOnScriptError(false);
			client.getOptions().setThrowExceptionOnFailingStatusCode(false);
			client.getOptions().setUseInsecureSSL(true);
			client.addRequestHeader("User-Agent", ua);
			client.waitForBackgroundJavaScript(600 * 1000);

			Thread.sleep(5000);

			page = client.getPage(url);

			client.close();
		} catch (MalformedURLException e) {
			e.printStackTrace();
		} catch (FailingHttpStatusCodeException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} catch (InterruptedException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

		html = page.asXml();
		doc = Jsoup.parse(html);
		return this;
	}

	public SimplePageDownloader download(URL URL) {
		return download(URL.toString());
	}

	/**
	 * 下载指定url的页面，默认返回Document，同时也提供纯文本形式的结果 默认followRedirect
	 * 
	 * @param url
	 * @return Document 下载结果
	 */
	public SimplePageDownloader download(String url) {
		CloseableHttpResponse response = null;
		try {
			this.url = UrlUtils.handleUrl(url);
			// Trust https certificates before openConnection
			trustAllHttpsCertificates();
			HttpsURLConnection.setDefaultHostnameVerifier(hv);

			HttpGet httpget = new HttpGet(this.url);
			Builder configBuilder = RequestConfig.custom();

			configBuilder.setConnectionRequestTimeout(connectionTimeout);
			configBuilder.setSocketTimeout(socketTimeout);
			configBuilder.setConnectTimeout(connectionTimeout);
			configBuilder.setCookieSpec(CookieSpecs.IGNORE_COOKIES);
			configBuilder.setCircularRedirectsAllowed(false);
			configBuilder.setMaxRedirects(2);

			if (useProxy()) {
				logger.info("Using proxy");
				HttpHost httpProxy = new HttpHost(p.poll());
				configBuilder.setProxy(httpProxy);
			}

			RequestConfig config = configBuilder.build();
			httpget.setConfig(config);
			httpget.addHeader("Accept-Encoding", "gzip");
			httpget.addHeader("User-Agent", ua);

			HttpClientContext context = HttpClientContext.create();
			response = httpclient.execute(httpget, context);
			status = response.getStatusLine().getStatusCode();

			html = getContent(charset, response);

			// 处理重定向，301/302以及meta重定向
			realUrl = realUrl(context, html);
			if (realUrl != null) {
				InputStream ins;
				byte[] contentBytes;
				httpget = new HttpGet(realUrl);
				response = httpclient.execute(httpget, context);
				ins = response.getEntity().getContent();
				contentBytes = IOUtils.toByteArray(ins);

				charset = getHtmlCharset(response, contentBytes);
				html = new String(contentBytes, charset);
			}

			// 判断如果response为空，则（代理）链接失败，换一个代理重试
			// int c = 0;
			// response = httpclient.execute(httpget);
			// while (response == null && p.size() > 0 && c < retry) {
			// logger.info("Retry + "+c+" Using proxy : "+p.poll());
			// try {
			// response = httpclient.execute(httpget);
			// c++;
			// }catch (org.apache.http.conn.HttpHostConnectException e) {
			//
			// }catch (org.apache.http.client.ClientProtocolException e) {
			//
			// }
			// }

			handleGzip(response);

			if (redirected())
				doc = Jsoup.parse(html, realUrl);
			else
				doc = Jsoup.parse(html, this.url);

		} catch (MalformedURLException e) {
			e.printStackTrace();
			logger.debug(e.getMessage() + " " + realUrl());
		} catch (ConnectTimeoutException e) {
			e.printStackTrace();
			logger.debug(e.getMessage() + " " + realUrl());
		} catch (ConnectException e) {
			e.printStackTrace();
			logger.debug(e.getMessage() + " " + realUrl());
		} catch (SocketException e) {
			e.printStackTrace();
			logger.debug(e.getMessage() + " " + realUrl());
		} catch (ClientProtocolException e) {
			e.printStackTrace();
			logger.debug(e.getMessage() + " " + realUrl());
		} catch (RedirectException e) {
			e.printStackTrace();
			logger.debug(e.getMessage() + " " + realUrl());
		} catch (EOFException e) {// 可能是gzip引起OEF异常，改用htmlunit下载
			logger.debug("Gzip Detected, Download by Simulating Browser: " + realUrl());
			return downloadByBrowse(url);
		} catch (Exception e) {
			logger.debug(e.getMessage() + " " + realUrl());
		} finally {
			if (realUrl() != null)
				logger.info(realUrl() + " Download Complete");
			try {
				if (response != null)
					EntityUtils.consume(response.getEntity());
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		return this;
	}

	public String changeCharset(String str, String newCharset) throws UnsupportedEncodingException {
		if (str != null) {
			// 用默认字符编码解码字符串。与系统相关，中文windows默认为GB2312
			byte[] bs = str.getBytes();
			return new String(bs, newCharset); // 用新的字符编码生成字符串
		}
		return null;
	}

	/**
	 * 获取真实的URL<br>
	 * 单独调用时使用，会新建连接<br>
	 * 首先检查是否301/302跳转，然后检查是否有meta跳转（百度常用）<br>
	 * 然后分别获取真实的URL<br>
	 * 301/302跳转从headers中的Location获取<br>
	 * meta跳转则需要下载页面，从meta中获取<br>
	 * 
	 * @param strUrl
	 * @return
	 */
	public String realUrl(String strUrl) {
		String url = null;
		int status;
		try {
			Connection con = Jsoup.connect(strUrl).header("User-Agent", ua);
			status = con.followRedirects(false).execute().statusCode();
			// 如果返回200，检查是否Meta redirects
			if (status == HttpURLConnection.HTTP_OK) {
				Elements meta = con.followRedirects(true).get().getElementsByTag("meta");
				for (Element e : meta) {
					for (Attribute a : e.attributes()) {
						if (a.getValue().equals("refresh")) {
							String b = e.attr("content");
							if (b.contains("\'"))
								url = b.substring(b.indexOf("\'") + 1, b.lastIndexOf("\'"));
						}
					}
				}
			} else if (status == HttpURLConnection.HTTP_MOVED_TEMP || status == HttpURLConnection.HTTP_MOVED_PERM
					|| status == HttpURLConnection.HTTP_SEE_OTHER) {
				url = con.execute().headers().get("Location");
			}
		} catch (MalformedURLException e) {
			url = strUrl;
		} catch (SocketTimeoutException e) {
			url = strUrl;
		} catch (IOException e) {
			url = strUrl;
		}

		if (url == null)
			url = strUrl;
		return url;
	}

	/**
	 * 获取真实的URL 可获取HTTP重定向后的URL，也可获取meta重定向
	 * 
	 * @param context
	 * @param html
	 * @return 真实的URL，如果没有重定向，返回空
	 */
	public String realUrl(HttpClientContext context, String html) {
		String realUrl;
		if (context.getRedirectLocations() != null && !context.getRedirectLocations().isEmpty()) {
			realUrl = context.getRedirectLocations().get(context.getRedirectLocations().size() - 1).toString();
			return realUrl;
		} else {
			realUrl = getMetaRedirects(html);
			return realUrl;
		}
	}

	/**
	 * 检查页面是否有meta重定向：<br>
	 * META http-equiv="refresh" content="0;URL=''"
	 * 
	 * @param html
	 * @return 重定向后的URL，如果没有重定向，返回空
	 */
	public String getMetaRedirects(String html) {
		Document doc = Jsoup.parse(html);
		Elements elements = doc.select("meta");
		String url = null;
		for (Element e : elements) {
			String content = e.attr("content");
			if (content.indexOf("URL=") != -1) {
				url = content.substring(content.indexOf("URL=\'"), content.length() - 1).split("=")[1].replace("\'",
						"");
				break;
			}
		}
		return url;
	}

	public String charset(String url) {
		String charset = null;
		try {
			HttpURLConnection con = (HttpURLConnection) new URL(realUrl(url)).openConnection();
			con.setRequestProperty("User-Agent", ua);
			charset = charset(con);
		} catch (IOException e) {
			e.printStackTrace();
		}
		return charset;
	}

	protected String getContent(String charset, HttpResponse httpResponse) throws IOException {
		if (charset == null) {
			byte[] contentBytes = IOUtils.toByteArray(httpResponse.getEntity().getContent());
			String htmlCharset = getHtmlCharset(httpResponse, contentBytes);
			if (htmlCharset != null) {
				return new String(contentBytes, htmlCharset);
			} else {
				logger.warn("Charset autodetect failed, use {} as charset. Please specify charset in Site.setCharset()",
						Charset.defaultCharset());
				return new String(contentBytes);
			}
		} else {
			return IOUtils.toString(httpResponse.getEntity().getContent(), charset);
		}
	}

	protected String getHtmlCharset(HttpResponse httpResponse, byte[] contentBytes) throws IOException {
		String charset = null;

		String value;
		try {
			value = httpResponse.getEntity().getContentType().getValue();
			charset = UrlUtils.getCharset(value);
			if (StringUtils.isNotBlank(charset)) {
				return charset;
			}
		} catch (NullPointerException e) {
			logger.debug("No charset in response headers");
		}

		// use default charset to decode first time
		Charset defaultCharset = Charset.defaultCharset();
		String content = new String(contentBytes, defaultCharset.name());
		// Charset in meta
		if (StringUtils.isNotEmpty(content)) {
			Document document = Jsoup.parse(content);
			Elements links = document.select("meta");
			for (Element link : links) {
				// 2.1 html4.01 <meta http-equiv="Content-Type"
				// content="text/html; charset=UTF-8" />
				String metaContent = link.attr("content");
				String metaCharset = link.attr("charset");
				if (metaContent.indexOf("charset") != -1) {
					metaContent = metaContent.substring(metaContent.indexOf("charset"), metaContent.length());
					charset = metaContent.split("=")[1];
					break;
				}
				// 2.2 html5 <meta charset="UTF-8" />
				else if (StringUtils.isNotEmpty(metaCharset)) {
					charset = metaCharset;
					break;
				}
			}
		}

		// 如果以上都不奏效，默认utf-8
		if (charset == null)
			charset = Charset.defaultCharset().name();
		return charset;
	}

	/**
	 * @deprecated
	 * @param conn
	 * @return
	 * @throws IOException
	 */
	public String charset(HttpURLConnection conn) throws IOException {
		Map<String, List<String>> map = conn.getHeaderFields();
		Set<String> keys = map.keySet();
		Iterator<String> iterator = keys.iterator();
		String strencoding = null;

		// 遍历,查找字符编码
		String key = null;
		String tmp = null;
		while (iterator.hasNext()) {
			key = iterator.next();
			tmp = map.get(key).toString().toLowerCase();
			// 获取content-type charset
			if (key != null && key.equals("Content-Type")) {
				int m = tmp.indexOf("charset=");
				if (m != -1) {
					strencoding = tmp.substring(m + 8).replace("]", "");
				}
			}
		}

		// 如果header没有，则打开InputStream，查看meta里是否有charset定义
		if (strencoding == null) {
			BufferedReader br = null;
			try {
				br = new BufferedReader(new InputStreamReader(conn.getInputStream()));
			} catch (FileNotFoundException e) {
				strencoding = "utf-8";
				return strencoding;
			} catch (ConnectException e) {
				strencoding = "utf-8";
				return strencoding;
			}

			String line;
			while ((line = br.readLine()) != null) {
				if (line.contains("charset") && line.contains("meta")) {
					Pattern p = Pattern.compile("(?<=charset=)\"?[^\">]*");
					Matcher m = p.matcher(line);
					if (m.find())
						strencoding = m.group(0).replaceAll("\"", "");
					break;
				}
			}
		}

		if (strencoding == null)
			strencoding = "utf-8";

		return strencoding;
	}

	private void handleGzip(HttpResponse httpResponse) {
		Header ceheader = httpResponse.getEntity().getContentEncoding();
		if (ceheader != null) {
			HeaderElement[] codecs = ceheader.getElements();
			for (HeaderElement codec : codecs) {
				if (codec.getName().equalsIgnoreCase("gzip")) {
					httpResponse.setEntity(new GzipDecompressingEntity(httpResponse.getEntity()));
				}
			}
		}
	}

	HostnameVerifier hv = new HostnameVerifier() {
		@Override
		public boolean verify(String urlHostName, SSLSession session) {
			logger.warn("Warning: URL Host: " + urlHostName + " vs. " + session.getPeerHost());
			return true;
		}
	};

	private static void trustAllHttpsCertificates() throws Exception {
		javax.net.ssl.TrustManager[] trustAllCerts = new javax.net.ssl.TrustManager[1];
		javax.net.ssl.TrustManager tm = new miTM();
		trustAllCerts[0] = tm;
		javax.net.ssl.SSLContext sc = javax.net.ssl.SSLContext.getInstance("TLS");
		sc.init(null, trustAllCerts, null);
		javax.net.ssl.HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
	}

	static class miTM implements javax.net.ssl.TrustManager, javax.net.ssl.X509TrustManager {
		@Override
		public java.security.cert.X509Certificate[] getAcceptedIssuers() {
			return null;
		}

		public boolean isServerTrusted(java.security.cert.X509Certificate[] certs) {
			return true;
		}

		public boolean isClientTrusted(java.security.cert.X509Certificate[] certs) {
			return true;
		}

		@Override
		public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType)
				throws java.security.cert.CertificateException {
			return;
		}

		@Override
		public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType)
				throws java.security.cert.CertificateException {
			return;
		}
	}

	class RetryHandler implements HttpRequestRetryHandler {
		@Override
		public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
			if (executionCount >= retry) {
				return false;
			}
			return true;
		}

	}
}
